{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aae3454714d9812",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-24T06:22:55.205567Z",
     "start_time": "2025-07-24T06:22:46.284575Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "ALBEF模型 - 包含交叉注意力机制\n",
      "================================================================================\n",
      "输入数据形状:\n",
      "  图像: torch.Size([4, 3, 224, 224])\n",
      "  文本ID: torch.Size([4, 8])\n",
      "  注意力掩码: torch.Size([4, 8])\n",
      "\n",
      "1. 单模态编码\n",
      "----------------------------------------\n",
      "  图像特征形状: torch.Size([4, 49, 768])\n",
      "  文本特征形状: torch.Size([4, 8, 768])\n",
      "\n",
      "2. 多模态编码（交叉注意力）\n",
      "----------------------------------------\n",
      "  多模态特征形状: torch.Size([4, 8, 768])\n",
      "  ✓ 文本特征已通过交叉注意力与图像特征融合\n",
      "\n",
      "3. 对比学习任务\n",
      "----------------------------------------\n",
      "  对比学习损失: 1.3920\n",
      "\n",
      "4. 图像-文本匹配任务（基于交叉注意力）\n",
      "----------------------------------------\n",
      "  ITM损失: 0.6441\n",
      "  ITM预测概率:\n",
      "    样本1: 不匹配=0.360, 匹配=0.640\n",
      "    样本2: 不匹配=0.223, 匹配=0.777\n",
      "    样本3: 不匹配=0.326, 匹配=0.674\n",
      "    样本4: 不匹配=0.469, 匹配=0.531\n",
      "\n",
      "5. 掩码语言模型任务（基于交叉注意力）\n",
      "----------------------------------------\n",
      "  MLM损失: 9.7902\n",
      "  ✓ MLM预测基于图像-文本交叉注意力特征\n",
      "\n",
      "6. 模型架构特点\n",
      "----------------------------------------\n",
      "  总参数量: 215,928,701\n",
      "  多模态编码器参数: 56,710,656\n",
      "  交叉注意力层数: 6\n",
      "  ✓ 包含图像-文本交叉注意力机制\n",
      "  ✓ 符合ALBEF论文架构\n",
      "================================================================================\n",
      "演示完成！现在的实现包含了论文中的交叉注意力机制\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from transformers import BertModel, BertTokenizer, BertConfig\n",
    "from torchvision.models import resnet50\n",
    "import random\n",
    "import numpy as np\n",
    "import math\n",
    "\n",
    "\n",
    "class ImageEncoder(nn.Module):\n",
    "    \"\"\"图像编码器（使用 ResNet50）\"\"\"\n",
    "    def __init__(self, embed_dim=768):  # 改为768以匹配BERT\n",
    "        super(ImageEncoder, self).__init__()\n",
    "        self.backbone = resnet50(pretrained=True)\n",
    "        # 移除最后的分类层，保留特征提取部分\n",
    "        self.backbone = nn.Sequential(*list(self.backbone.children())[:-2])\n",
    "        \n",
    "        # 添加自适应池化和投影层\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((7, 7))  # 输出 7x7 特征图\n",
    "        self.proj = nn.Linear(2048, embed_dim)  # ResNet50的输出通道数是2048\n",
    "        \n",
    "    def forward(self, images):\n",
    "        # images: (batch_size, 3, 224, 224)\n",
    "        features = self.backbone(images)  # (batch_size, 2048, 7, 7)\n",
    "        features = self.avgpool(features)  # (batch_size, 2048, 7, 7)\n",
    "        \n",
    "        # 将空间维度展平，保留patch信息\n",
    "        batch_size, channels, h, w = features.shape\n",
    "        features = features.view(batch_size, channels, h * w).transpose(1, 2)  # (batch_size, 49, 2048)\n",
    "        \n",
    "        # 投影到BERT维度\n",
    "        features = self.proj(features)  # (batch_size, 49, 768)\n",
    "        \n",
    "        return features\n",
    "\n",
    "\n",
    "class TextEncoder(nn.Module):\n",
    "    \"\"\"文本编码器（使用 BERT）\"\"\"\n",
    "    def __init__(self, embed_dim=768):\n",
    "        super(TextEncoder, self).__init__()\n",
    "        self.bert = BertModel.from_pretrained('bert-base-uncased')\n",
    "        self.embed_dim = embed_dim\n",
    "\n",
    "    def forward(self, input_ids, attention_mask):\n",
    "        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        return outputs.last_hidden_state  # (batch_size, seq_len, 768)\n",
    "\n",
    "\n",
    "class MultiHeadCrossAttention(nn.Module):\n",
    "    \"\"\"多头交叉注意力机制\"\"\"\n",
    "    def __init__(self, embed_dim=768, num_heads=12):\n",
    "        super(MultiHeadCrossAttention, self).__init__()\n",
    "        self.embed_dim = embed_dim\n",
    "        self.num_heads = num_heads\n",
    "        self.head_dim = embed_dim // num_heads\n",
    "        \n",
    "        assert self.head_dim * num_heads == embed_dim, \"embed_dim must be divisible by num_heads\"\n",
    "        \n",
    "        self.q_linear = nn.Linear(embed_dim, embed_dim)\n",
    "        self.k_linear = nn.Linear(embed_dim, embed_dim)\n",
    "        self.v_linear = nn.Linear(embed_dim, embed_dim)\n",
    "        self.out = nn.Linear(embed_dim, embed_dim)\n",
    "        \n",
    "    def forward(self, query, key, value, attention_mask=None):\n",
    "        batch_size = query.size(0)\n",
    "        \n",
    "        # 线性变换并重塑为多头格式\n",
    "        Q = self.q_linear(query).view(batch_size, -1, self.num_heads, self.head_dim).transpose(1, 2)\n",
    "        K = self.k_linear(key).view(batch_size, -1, self.num_heads, self.head_dim).transpose(1, 2)\n",
    "        V = self.v_linear(value).view(batch_size, -1, self.num_heads, self.head_dim).transpose(1, 2)\n",
    "        \n",
    "        # 计算注意力\n",
    "        attention = self._attention(Q, K, V, attention_mask)\n",
    "        \n",
    "        # 合并多头\n",
    "        attention = attention.transpose(1, 2).contiguous().view(\n",
    "            batch_size, -1, self.embed_dim)\n",
    "        \n",
    "        return self.out(attention)\n",
    "    \n",
    "    def _attention(self, Q, K, V, attention_mask=None):\n",
    "        scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.head_dim)\n",
    "        \n",
    "        if attention_mask is not None:\n",
    "            # 扩展mask到多头维度\n",
    "            attention_mask = attention_mask.unsqueeze(1).unsqueeze(1)\n",
    "            scores = scores.masked_fill(attention_mask == 0, -1e9)\n",
    "        \n",
    "        attention_weights = F.softmax(scores, dim=-1)\n",
    "        attention = torch.matmul(attention_weights, V)\n",
    "        \n",
    "        return attention\n",
    "\n",
    "\n",
    "class MultimodalEncoderLayer(nn.Module):\n",
    "    \"\"\"多模态编码器层 - 包含自注意力和交叉注意力\"\"\"\n",
    "    def __init__(self, embed_dim=768, num_heads=12, ff_dim=3072, dropout=0.1):\n",
    "        super(MultimodalEncoderLayer, self).__init__()\n",
    "        \n",
    "        # 自注意力层\n",
    "        self.self_attention = nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout, batch_first=True)\n",
    "        \n",
    "        # 交叉注意力层（文本查询图像）\n",
    "        self.cross_attention = MultiHeadCrossAttention(embed_dim, num_heads)\n",
    "        \n",
    "        # 前馈网络\n",
    "        self.feed_forward = nn.Sequential(\n",
    "            nn.Linear(embed_dim, ff_dim),\n",
    "            nn.GELU(),\n",
    "            nn.Dropout(dropout),\n",
    "            nn.Linear(ff_dim, embed_dim),\n",
    "            nn.Dropout(dropout)\n",
    "        )\n",
    "        \n",
    "        # Layer Normalization\n",
    "        self.norm1 = nn.LayerNorm(embed_dim)\n",
    "        self.norm2 = nn.LayerNorm(embed_dim)\n",
    "        self.norm3 = nn.LayerNorm(embed_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, text_features, image_features, text_attention_mask=None):\n",
    "        # 1. 文本自注意力\n",
    "        residual = text_features\n",
    "        text_features = self.norm1(text_features)\n",
    "        \n",
    "        # 创建文本的key_padding_mask\n",
    "        if text_attention_mask is not None:\n",
    "            key_padding_mask = (text_attention_mask == 0)\n",
    "        else:\n",
    "            key_padding_mask = None\n",
    "            \n",
    "        text_attn_out, _ = self.self_attention(\n",
    "            text_features, text_features, text_features,\n",
    "            key_padding_mask=key_padding_mask\n",
    "        )\n",
    "        text_features = residual + self.dropout(text_attn_out)\n",
    "        \n",
    "        # 2. 交叉注意力（文本查询图像）\n",
    "        residual = text_features\n",
    "        text_features = self.norm2(text_features)\n",
    "        \n",
    "        cross_attn_out = self.cross_attention(\n",
    "            query=text_features,\n",
    "            key=image_features,\n",
    "            value=image_features\n",
    "        )\n",
    "        text_features = residual + self.dropout(cross_attn_out)\n",
    "        \n",
    "        # 3. 前馈网络\n",
    "        residual = text_features\n",
    "        text_features = self.norm3(text_features)\n",
    "        ff_out = self.feed_forward(text_features)\n",
    "        text_features = residual + ff_out\n",
    "        \n",
    "        return text_features\n",
    "\n",
    "\n",
    "class MultimodalEncoder(nn.Module):\n",
    "    \"\"\"多模态编码器 - 包含多层交叉注意力\"\"\"\n",
    "    def __init__(self, num_layers=6, embed_dim=768, num_heads=12, ff_dim=3072, dropout=0.1):\n",
    "        super(MultimodalEncoder, self).__init__()\n",
    "        self.layers = nn.ModuleList([\n",
    "            MultimodalEncoderLayer(embed_dim, num_heads, ff_dim, dropout)\n",
    "            for _ in range(num_layers)\n",
    "        ])\n",
    "        \n",
    "    def forward(self, text_features, image_features, text_attention_mask=None):\n",
    "        # 通过多层多模态编码器\n",
    "        for layer in self.layers:\n",
    "            text_features = layer(text_features, image_features, text_attention_mask)\n",
    "        \n",
    "        return text_features\n",
    "\n",
    "\n",
    "class ALBEF(nn.Module):\n",
    "    \"\"\"ALBEF模型 - 包含交叉注意力的完整实现\"\"\"\n",
    "    def __init__(self, embed_dim=768, num_multimodal_layers=6):\n",
    "        super(ALBEF, self).__init__()\n",
    "        \n",
    "        # 单模态编码器\n",
    "        self.image_encoder = ImageEncoder(embed_dim)\n",
    "        self.text_encoder = TextEncoder(embed_dim)\n",
    "        \n",
    "        # 多模态编码器（关键组件）\n",
    "        self.multimodal_encoder = MultimodalEncoder(\n",
    "            num_layers=num_multimodal_layers,\n",
    "            embed_dim=embed_dim\n",
    "        )\n",
    "        \n",
    "        # 用于对比学习的投影头\n",
    "        self.image_proj = nn.Linear(embed_dim, embed_dim)\n",
    "        self.text_proj = nn.Linear(embed_dim, embed_dim)\n",
    "        \n",
    "        # 温度参数\n",
    "        self.temperature = nn.Parameter(torch.ones([]) * 0.07)\n",
    "        \n",
    "        # ITM任务头\n",
    "        self.itm_head = nn.Linear(embed_dim, 2)\n",
    "        \n",
    "        # MLM任务头\n",
    "        self.mlm_head = nn.Linear(embed_dim, 30522)  # BERT词汇表大小\n",
    "        \n",
    "    def forward(self, images, input_ids, attention_mask, task=\"contrastive\", **kwargs):\n",
    "        if task == \"contrastive\":\n",
    "            return self.contrastive_forward(images, input_ids, attention_mask)\n",
    "        elif task == \"itm\":\n",
    "            return self.itm_forward(images, input_ids, attention_mask, **kwargs)\n",
    "        elif task == \"mlm\":\n",
    "            return self.mlm_forward(images, input_ids, attention_mask, **kwargs)\n",
    "        else:\n",
    "            raise ValueError(f\"不支持的任务类型: {task}\")\n",
    "    \n",
    "    def encode_unimodal(self, images, input_ids, attention_mask):\n",
    "        \"\"\"编码单模态特征\"\"\"\n",
    "        # 图像编码\n",
    "        image_features = self.image_encoder(images)  # (batch_size, 49, 768)\n",
    "        \n",
    "        # 文本编码  \n",
    "        text_features = self.text_encoder(input_ids, attention_mask)  # (batch_size, seq_len, 768)\n",
    "        \n",
    "        return image_features, text_features\n",
    "    \n",
    "    def encode_multimodal(self, images, input_ids, attention_mask):\n",
    "        \"\"\"编码多模态特征（通过交叉注意力）\"\"\"\n",
    "        # 获取单模态特征\n",
    "        image_features, text_features = self.encode_unimodal(images, input_ids, attention_mask)\n",
    "        \n",
    "        # 通过多模态编码器进行交叉注意力融合\n",
    "        multimodal_text_features = self.multimodal_encoder(\n",
    "            text_features, image_features, attention_mask\n",
    "        )\n",
    "        \n",
    "        return multimodal_text_features, image_features\n",
    "    \n",
    "    def contrastive_forward(self, images, input_ids, attention_mask):\n",
    "        \"\"\"对比学习前向传播\"\"\"\n",
    "        # 使用单模态编码器进行对比学习\n",
    "        image_features, text_features = self.encode_unimodal(images, input_ids, attention_mask)\n",
    "        \n",
    "        # 池化操作获取全局特征,做到2维\n",
    "        image_embeds = image_features.mean(dim=1)  # (batch_size, 768)\n",
    "        text_embeds = text_features[:, 0, :]  # 使用[CLS] token\n",
    "        \n",
    "        # 投影到对比学习空间\n",
    "        image_embeds = self.image_proj(image_embeds)\n",
    "        text_embeds = self.text_proj(text_embeds)\n",
    "        \n",
    "        # 归一化\n",
    "        image_embeds = F.normalize(image_embeds, dim=-1)\n",
    "        text_embeds = F.normalize(text_embeds, dim=-1)\n",
    "        \n",
    "        # 计算对比损失\n",
    "        logits = torch.matmul(image_embeds, text_embeds.t()) / self.temperature\n",
    "        labels = torch.arange(logits.size(0), device=logits.device)\n",
    "        \n",
    "        loss_i = F.cross_entropy(logits, labels)\n",
    "        loss_t = F.cross_entropy(logits.t(), labels)\n",
    "        loss = (loss_i + loss_t) / 2\n",
    "        \n",
    "        return loss\n",
    "    \n",
    "    def itm_forward(self, images, input_ids, attention_mask, labels=None):\n",
    "        \"\"\"图像-文本匹配前向传播\"\"\"\n",
    "        # 使用多模态编码器\n",
    "        multimodal_features, _ = self.encode_multimodal(images, input_ids, attention_mask)\n",
    "        \n",
    "        # 使用[CLS] token进行分类\n",
    "        cls_features = multimodal_features[:, 0, :]  # (batch_size, 768)\n",
    "        itm_logits = self.itm_head(cls_features)  # (batch_size, 2)\n",
    "        \n",
    "        if labels is not None:\n",
    "            loss = F.cross_entropy(itm_logits, labels)\n",
    "            return loss\n",
    "        else:\n",
    "            return itm_logits\n",
    "    \n",
    "    def mlm_forward(self, images, input_ids, attention_mask, masked_input_ids=None, mlm_labels=None):\n",
    "        \"\"\"掩码语言模型前向传播\"\"\"\n",
    "        if masked_input_ids is None and mlm_labels is None:\n",
    "            masked_input_ids, mlm_labels = self.mask_tokens(input_ids)\n",
    "        \n",
    "        # 使用多模态编码器处理掩码文本\n",
    "        multimodal_features, _ = self.encode_multimodal(images, masked_input_ids, attention_mask)\n",
    "        \n",
    "        # MLM预测\n",
    "        mlm_logits = self.mlm_head(multimodal_features)  # (batch_size, seq_len, vocab_size)\n",
    "        \n",
    "        if mlm_labels is not None:\n",
    "            loss = F.cross_entropy(\n",
    "                mlm_logits.view(-1, mlm_logits.size(-1)),\n",
    "                mlm_labels.view(-1),\n",
    "                ignore_index=-100\n",
    "            )\n",
    "            return loss\n",
    "        else:\n",
    "            return mlm_logits\n",
    "    \n",
    "    def mask_tokens(self, input_ids, mlm_probability=0.15):\n",
    "        \"\"\"创建用于MLM任务的掩码输入和标签\"\"\"\n",
    "        device = input_ids.device\n",
    "        labels = input_ids.clone()\n",
    "        \n",
    "        # 创建掩码概率矩阵\n",
    "        probability_matrix = torch.full(labels.shape, mlm_probability, device=device)\n",
    "        \n",
    "        # 特殊token不应该被掩码\n",
    "        special_tokens_mask = torch.zeros_like(input_ids, dtype=torch.bool, device=device)\n",
    "        for special_id in [0, 101, 102]:  # [PAD], [CLS], [SEP]\n",
    "            special_tokens_mask = special_tokens_mask | (input_ids == special_id)\n",
    "        probability_matrix.masked_fill_(special_tokens_mask, value=0.0)\n",
    "        \n",
    "        # 创建掩码\n",
    "        masked_indices = torch.bernoulli(probability_matrix).bool()\n",
    "        labels[~masked_indices] = -100\n",
    "        \n",
    "        # 80%的情况下用[MASK]替换\n",
    "        indices_replaced = torch.bernoulli(torch.full(labels.shape, 0.8, device=device)).bool() & masked_indices\n",
    "        input_ids[indices_replaced] = 103  # [MASK] token ID\n",
    "        \n",
    "        # 10%的情况下用随机token替换\n",
    "        indices_random = torch.bernoulli(torch.full(labels.shape, 0.5, device=device)).bool() & masked_indices & ~indices_replaced\n",
    "        random_words = torch.randint(0, 30522, labels.shape, device=device)\n",
    "        input_ids[indices_random] = random_words[indices_random]\n",
    "        \n",
    "        return input_ids, labels\n",
    "\n",
    "\n",
    "def demo_albef_with_cross_attention():\n",
    "    \"\"\"演示包含交叉注意力的ALBEF模型\"\"\"\n",
    "    print(\"=\" * 80)\n",
    "    print(\"ALBEF模型 - 包含交叉注意力机制\")\n",
    "    print(\"=\" * 80)\n",
    "    \n",
    "    # 初始化模型\n",
    "    model = ALBEF(embed_dim=768, num_multimodal_layers=6)\n",
    "    model.eval()\n",
    "    \n",
    "    # 准备示例数据\n",
    "    from transformers import BertTokenizer\n",
    "    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "    \n",
    "    texts = [\n",
    "        \"A cat sitting on a chair\",\n",
    "        \"A dog running in the park\", \n",
    "        \"A bird flying in the sky\",\n",
    "        \"A fish swimming in water\"\n",
    "    ]\n",
    "    \n",
    "    tokenized = tokenizer(texts, padding=True, truncation=True, return_tensors='pt', max_length=32)\n",
    "    images = torch.randn(4, 3, 224, 224)\n",
    "    \n",
    "    print(f\"输入数据形状:\")\n",
    "    print(f\"  图像: {images.shape}\")\n",
    "    print(f\"  文本ID: {tokenized['input_ids'].shape}\")\n",
    "    print(f\"  注意力掩码: {tokenized['attention_mask'].shape}\")\n",
    "    print()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        # 1. 单模态编码\n",
    "        print(\"1. 单模态编码\")\n",
    "        print(\"-\" * 40)\n",
    "        image_features, text_features = model.encode_unimodal(\n",
    "            images, tokenized['input_ids'], tokenized['attention_mask']\n",
    "        )\n",
    "        print(f\"  图像特征形状: {image_features.shape}\")  # (4, 49, 768)\n",
    "        print(f\"  文本特征形状: {text_features.shape}\")   # (4, seq_len, 768)\n",
    "        print()\n",
    "        \n",
    "        # 2. 多模态编码（交叉注意力）\n",
    "        print(\"2. 多模态编码（交叉注意力）\")\n",
    "        print(\"-\" * 40)\n",
    "        multimodal_features, _ = model.encode_multimodal(\n",
    "            images, tokenized['input_ids'], tokenized['attention_mask']\n",
    "        )\n",
    "        print(f\"  多模态特征形状: {multimodal_features.shape}\")  # (4, seq_len, 768)\n",
    "        print(\"  ✓ 文本特征已通过交叉注意力与图像特征融合\")\n",
    "        print()\n",
    "        \n",
    "        # 3. 对比学习任务\n",
    "        print(\"3. 对比学习任务\")\n",
    "        print(\"-\" * 40)\n",
    "        contrastive_loss = model(images, tokenized['input_ids'], tokenized['attention_mask'], task=\"contrastive\")\n",
    "        print(f\"  对比学习损失: {contrastive_loss.item():.4f}\")\n",
    "        print()\n",
    "        \n",
    "        # 4. ITM任务（使用多模态特征）\n",
    "        print(\"4. 图像-文本匹配任务（基于交叉注意力）\")\n",
    "        print(\"-\" * 40)\n",
    "        itm_labels = torch.tensor([1, 1, 0, 0])\n",
    "        itm_loss = model(images, tokenized['input_ids'], tokenized['attention_mask'], \n",
    "                        task=\"itm\", labels=itm_labels)\n",
    "        print(f\"  ITM损失: {itm_loss.item():.4f}\")\n",
    "        \n",
    "        itm_logits = model.itm_forward(images, tokenized['input_ids'], tokenized['attention_mask'])\n",
    "        itm_predictions = torch.softmax(itm_logits, dim=-1)\n",
    "        print(f\"  ITM预测概率:\")\n",
    "        for i, pred in enumerate(itm_predictions):\n",
    "            print(f\"    样本{i+1}: 不匹配={pred[0]:.3f}, 匹配={pred[1]:.3f}\")\n",
    "        print()\n",
    "        \n",
    "        # 5. MLM任务（使用多模态特征）\n",
    "        print(\"5. 掩码语言模型任务（基于交叉注意力）\")\n",
    "        print(\"-\" * 40)\n",
    "        mlm_loss = model(images, tokenized['input_ids'], tokenized['attention_mask'], task=\"mlm\")\n",
    "        print(f\"  MLM损失: {mlm_loss.item():.4f}\")\n",
    "        print(\"  ✓ MLM预测基于图像-文本交叉注意力特征\")\n",
    "        print()\n",
    "        \n",
    "        # 6. 模型架构对比\n",
    "        print(\"6. 模型架构特点\")\n",
    "        print(\"-\" * 40)\n",
    "        total_params = sum(p.numel() for p in model.parameters())\n",
    "        multimodal_params = sum(p.numel() for p in model.multimodal_encoder.parameters())\n",
    "        \n",
    "        print(f\"  总参数量: {total_params:,}\")\n",
    "        print(f\"  多模态编码器参数: {multimodal_params:,}\")\n",
    "        print(f\"  交叉注意力层数: {len(model.multimodal_encoder.layers)}\")\n",
    "        print(f\"  ✓ 包含图像-文本交叉注意力机制\")\n",
    "        print(f\"  ✓ 符合ALBEF论文架构\")\n",
    "        \n",
    "    print(\"=\" * 80)\n",
    "    print(\"演示完成！现在的实现包含了论文中的交叉注意力机制\")\n",
    "    print(\"=\" * 80)\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    torch.manual_seed(42)\n",
    "    np.random.seed(42)\n",
    "    demo_albef_with_cross_attention()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "db5b99ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "#transformers库中没有ALBEF模型\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
